# -- encoding:utf-8 --

import numpy as np
import pandas as pd
import re


# 清洗字符串，字符切分
def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    if not isinstance(string, str):
        if np.isnan(string):
            # 如果为空值改为"<UNK>"
            string = "<UNK>"
        else:
            string = str(string)

    string = re.sub(r"[^\u4e00-\u9fa5A-Za-z0-9(),.!?，。？！、“”\'\`]", " ", string)  # 考虑到中文
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()


def load_data_and_labels(csv_data_file):
    """
    基于给定CSV格式的文件加载数据内容
    :param csv_data_file:
    :return:
    """
    # 1. 加载所有数据组成DataFrame
    df = pd.read_csv(csv_data_file, sep=",", encoding='utf-8')

    # 2.获取DataFrame中的文本字符串以及标签值
    texts = np.asarray([clean_str(sentence) for sentence in df['review']])
    labels = np.asarray(df['label'], dtype=np.int32)

    # 6. 结果返回
    return texts, labels


def batch_iter(data, batch_size, num_epochs, shuffle=True):
    """
    基于给定的data数据获取批次数据
    :param data:
    :param batch_size:
    :param num_epochs:
    :param shuffle:
    :return:
    """
    data = np.array(data)
    data_size = len(data)
    # 一个epoch里面有多少个bachsize
    num_batches_per_epoch = int((len(data) - 1) / batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        if shuffle:
            # 传给permutation一个矩阵，它会返回一个洗牌后的矩阵副本
            shuffle_indices = np.random.permutation(np.arange(data_size))
            shuffled_data = data[shuffle_indices]
        else:
            shuffled_data = data
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]


if __name__ == '__main__':
    # 1. 加载数据
    texts, labels = load_data_and_labels(
        # csv_data_file="../data/ChnSentiCorp_htl_all.csv"
        csv_data_file="../data/simplifywenshu_4_moods.csv"
    )
    from utils.vocabulary_utils import VocabularyProcessorUtil, split_with_word

    # 2. 加载训练好的Embedding Table中所携带的单词词向量转换矩阵
    # _, vocabulary = VocabularyProcessorUtil.load_word2vec_embedding("../hotel/model/w2v.bin")
    _, vocabulary = VocabularyProcessorUtil.load_word2vec_embedding("../wenshu/model/w2v.bin")
    # 3. 词汇转换模型估计并持久化
    VocabularyProcessorUtil.building_model(
        documents=texts,
        # save_path='../hotel/model/vocab.pkl',
        save_path='../wenshu/model/vocab.pkl',
        max_document_length=512,  # 最好将所有训练数据的长度统计一下
        vocabulary=vocabulary,
        split_fn=split_with_word)

    # # 4. 词汇转换模型加载应用测试(可以注释掉)
    # model = VocabularyProcessorUtil.load_model('../wenshu/model/vocab.pkl')
    # r = model.transform(['今天手机报上那个王宝强的哪吒造型，彻底笑喷了', '离婚不离婚应该是当事人的感受'])
    # r = np.asarray(list(r))
    # print(r.shape)
    # print(r)
    # print("词汇数目:{}".format(len(model.vocabulary_)))
